{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "0PIghxafQ6R8",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 437.0
    },
    "outputId": "92e0fb28-7307-4a92-d601-980997471481"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1000 cost = 4.152723\n",
      "Epoch: 2000 cost = 3.183771\n",
      "Epoch: 3000 cost = 2.936138\n",
      "Epoch: 4000 cost = 2.803021\n",
      "Epoch: 5000 cost = 3.062087\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAecAAAFKCAYAAAAnj5dkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl0U3X+//Fnkqal0DVtKgrqyCaI\ngOULKgKDVFDcEIXa4jLqMG4I6ogim19QARdUdARlEQFRAQuoiAuyK/uPb5FNx8KggoDS0jZtaWna\nJL8/GCKVUmhpm0vu63HOnOnN5+be99t4feUuudfi8/l8iIiIiGFYA12AiIiIlKVwFhERMRiFs4iI\niMEonEVERAxG4SwiImIwCmcRERGDCQl0AcdkZuYHuoQTxMbWJSenMNBl1Aqz9GqWPkG9BiOz9Anm\n6NXpjDzpmPacKxASYgt0CbXGLL2apU9Qr8HILH2CuXotj8JZRETEYIIunCdNmsAnn8wLdBkiIiJV\nZphzztXloYcGBLoEERGRMxKwcP71119JTU3lnnvuYd68eZSWehkx4llmznyHnTszuPzyKxk2bCTL\nly9l+vQpeDwe4uOdPP30CNxuN/37/4PPPvuakJCjLQwdOogrrujAjh3badCgIffe+w9++mk3r776\nIllZWYSG2hk2bCTNm18SqJZFREROS0APa+fk5OB0Olm8eDFNmjRh5MihDB/+LDNnzmbp0sVs3vx/\nvPzyaF544VU+/HA+HTp04uWXx3LRRY2Ii4tj69bvADhy5Aj/93+b6NLlGv+yvV4vQ4c+SY8eNzBn\nzgKefHIoQ4YMorS0NFDtioiInJaAhnNpaSk9evQAoFGjJrRocQkxMTFER8cQFxfP5s3/R2JiOxo2\nPB+Am2/uxebNmygtLeXqq69h9epVAGzYsJYWLVoSGxvrX/Yvv/xMbm42N954CwCtW19GTEws27dv\nreUuRUREKieg55xtNht16tQBwGq1Eh5e1z9mtVqx20OJjPzjd2ARERH4fD5crlyuvvoahg17kkcf\nHcQ336zkmmu6l1l2QUE+R44c4c47+/hfO3z4MC6Xq4a7EhEROTOGviDM4XCwY8cfe7p5eXlYrVb/\nnrXVamPnzgw2blzPo48+Uea98fFO6tWrx4cfzq/tskVERM6IoX9KFRISwnffbWbfvl8B+PTT+bRv\nf4X/IrCuXa/h3Xen0LRpM6KjY8q8t379c3E6z2HFiqUA5ObmMnLkMIqKimq3CRERkUoy9J6z05nA\nkCEjGDr06IVc557bgMGDh/nHr776Gvr1u4shQ0ac8F6LxcKzz45l3LixTJ36NlarlZSUOwkPD6/N\nFkRERCrN4vP5fIEuAmrm3trFJR5cBcVER4QRZq/8reCczkhD3vO7JpilV7P0Ceo1GJmlTzBHrxXd\nW9vQe85V5fF6mbt8F5szMsnOK8YRFUZiMycpSU2wWQ19JF9ERCQ4w3nu8l0s3fSrf/pQXrF/+o5u\nzQJVloiIyGkJut3I4hIPmzMyyx3bnJFFcYmnlisSERGpnKALZ1dBMdl5xeWO5eQfwVVQ/piIiIhR\nVDmcx44dS0pKCqmpqWzdWvauW2vXrqVPnz6kpKQwceLEMy6yMqIjwnBEhZU7FhtZh+iI8sdERESM\nokrhvHHjRn755Rfmzp3LmDFjGDNmTJnx0aNH8+abbzJ79mzWrFnDrl27qqXY0xFmt5HYzFnuWGKz\n+CpdtS0iIlKbqhTO69ato1u3bgA0btwYl8tFQUEBAHv37iU6Oppzzz0Xq9VKly5dWLduXfVVfBpS\nkprQrV1D4qLqYLVAXFQdurVrSEpSk1qtQ0REpCqqdLV2VlYWLVu29E87HA4yMzOJiIggMzMTh8NR\nZmzv3r1nXmkl2KxW7ujWjN5dGp/R75xFREQCoVp+SlUd9zGJja1LSEj1B2jDM3x/RT8SDzZm6dUs\nfYJ6DUZm6RPM1eufVSmcExISyMrK8k8fPHgQp9NZ7tjvv/9OQkLCKZeZk1NYlVJqlBnuUHOMWXo1\nS5+gXoORWfoEc/Ra0ZePKp1z7tixI4sXLwZgx44dJCQkEBERAUDDhg0pKCjg119/pbS0lBUrVtCx\nY8eqrEZERMSUqrTn3LZtW1q2bElqaioWi4WRI0eyYMECIiMj6d69O6NGjWLQoEEA3HDDDVx00UXV\nWrSIiEgwC+oHX5wpMxxWOcYsvZqlT1CvwcgsfYI5eq32w9oiYjzp6ZtISelVLcv64ovPeOyx/tWy\nLBGpPIWziIiIwSicRYLMhAmvk5p6Gz169GDbti0UFxczbtxY+va9jTvv7MObb47H4zn6AJhdu3by\n8MN/5447enPvvXewYcOJNww6fLiAu+++nRUrltZ2KyKmFZSPjBQxq99+O0Dz5i0YMOBxVqz4ktde\ne4mkpGs5ePB3Zs36CI+nlAEDHmDp0sV0796DUaOGcc89/ejevQf//vf3/POfA5g//zP/8rxeL88+\nO4Jrr72erl27BbAzEXPRnrNIEAkNDSUpqTsA119/PTt3ZrBq1XJ69ryVkJAQwsLq0L379WzcuJ4D\nB/Zz6NAhunW7DoDmzS+hfv36/PDD9/7lTZo0gZiYWO6++76A9CNiVgpnkSASFRWN1Xp0sz5274HD\nhwuIjIzyzxMZGUlOTg45OTlERERisViOG4siJycbgB9//J758+fidJ76JkIiUr0UziJBJD//j5+e\n5OXlAUcD2+VyHfe6C4fDgcPhID/fVeb2uy6XC4cjDoD4eCfvvTeXhQs/5ocfdtRSByICCmeRoFJc\nfIRVq1YAsHjxYlq0uIQuXbry+eef4vF4KCoqYvHiL+jQoRPnnnseTmcCy5Z9DcC2bVvIzj5EixZH\nH2oTF+ekQYOGDBz4T8aMGUVxcXHA+hIxG10QJhJELrjgQnbs2MrkyROw20MYMuR/adSoCfv37+Pu\nu2/HYrHQtWs3kpK6YbFYePbZsYwb9wLTp0+lTp1wnn/+RcLDw8ss89prr2flyuVMmfIWAwf+M0Cd\niZiL7hBWATPcoeYYs/Rqlj7hzHr1ekvwlORjs0ditdqrubLqZ5bP1Sx9gjl6regOYdpzFhE/n89L\nzr6vKcr9EU+JC5s9mvCYi4ltcC0Wi86CidQWhbOI+OXs+5qCzI3+aU+Jyz/taNgjUGWJmI6+CosI\ncPRQdlHuj+WOFeVm4PWW1HJFIualcBYxiUmTJvDJJ/NOOu4pycdT4jrJmAtPSXCf/xMxEh3WFjGJ\nhx4aUOG4zR6JzR5dbkDb7NHY7Ce/eEVEqpf2nEUM5sCB/dxyy3V88MFM+va9jdTU29i+fRtPPfUY\nvXpdz9ixz57weMjjp3fv3sWDD97HjTfeSGrqrcyfPxeAMWNGMWPGOwD8+98/8Pe/30Vq6m0MGPAA\n+/fvw2q1Ex5zcbk1hcc0Oyuu2hYJFgpnEQPKzc3F4Yhj9uwFNGnShJEjhzJ8+LPMnDmbpUsXs3//\nryd977vvTqVXr958/vnnTJo0nU2bNuJ2u8vMM2rUMO6//2HmzFnAX/96NePHvwxAbINriXBejs0e\nA1iw2WOIcF5ObINra7JdEfkTHdYWMSCPx0NS0tGnQDVq1ASAmJgYAOLi4snKyjrpe2NjHaxcuYz/\n+Z/WOJ3n88ILr5YZ37PnF3Jzc+nQoSMAt912O7fc0hsAi8WKo2EPvOddc1b9zlkk2GjPWcSAbDYb\nYWF1ALBarYSH1/WPWa1WWre+7KTvffjhgTRq1ITHH3+c2267kQUL0sqMu1y5/odiAP99WlVYmXms\nVjv2MIeCWSRAFM4iZ6GtW7/D6/X6p49/4EXdunV58MFHWLJkCWPHjuOddyaxZ88v/vHo6Bjy8vL8\n7y8tLeXAgf21V7yInJLCWeQsFBcXz6FDWeTkZOPxeFiy5Ev/2ODB/2T37v8A0KhRY+rViyjzWMjz\nz7+AhIQEVq1aDsCiRZ/w8stjarcBEamQzjmLnIUaNGjIDTf05L777uScc+rTo8eN7NyZAUCfPik8\n++wIfD4PHo+XW2/tw/nnX+B/r8Vi4bnnXuT5559h8uSJxMXFM2zYyEC1IiLl0IMvKmCGG68fY5Ze\ng73P5557hu++S+fgwd9ZsGABCQkXlDvfgQP7SU29lVWrNtRyhTUj2D/XY8zSJ5ijVz34QsQkli5d\nzOzZC2jQoGGF/3Er8ZTg8/lwe9yE2kJruUoRORWFs0iQGDDgAbxeL088MYDDhw8zceIEGjRozCuv\nvMCWLZvxer00atSElqlX8N2eLXjx0f/NgWSu34u1GPr3f5Tu3fVwCxEj0AVhIkFiwoQpALz55mTq\n1Dn6M6yNG9dz4MB+PvxwPnPmfExxtIfF67/C5XaBz0d+cQENH7iU9sl/ZerUtwNZvogcR+EsEsRi\nYmL4+efdfPPNCvIO51GvcwJRTeOODvrA0aY+ANmR+WRmHgxgpSJyPIWzSBC75JJLefzxp5g3by69\nb72Jze+vxlP030c/Wi1YQ20AuNz5eDyeAFYqIsdTOIsEua5du/Hmm5OZ89HHhHisHFyz54R5YsKi\n4LjfQotIYCmcRYLY558v9D+JKj42ngsvvKjc+Zo7mqFoFjEOhbNIEOvcuQs//vgDqam3cuedfQhz\n2bi1z+3EhEUDEFcnlqsbduK6C7sGuFIROZ5uQlIBM/wI/hiz9GqWPqHiXt0eNzn5h6h3xEvd2His\nf3rwxdnGLJ+rWfoEc/Sqm5CIiJ/P4yE3LY3Dm9NxZWcT4nAQkdgWZ3IqFpst0OWJCApnEdPJTJtD\n7tIl/unSQ4f80wmpdwaqLBE5js45i5iIt7iYgs3p5Y4VbN6Mt7i4lisSkfIonEVMpNTlojQ7u/yx\nnGxKXa5arkhEyqNwFjGRkOhoQhyO8sdiHYRER9dyRSJSHoWziIlYw8KISGxb7lhEYuJZf9W2SLDQ\nBWEiJuNMTgWOnmMuzckmJNZBRGKi/3URCTyFs4jJWGw2ElLvJP7WPpS6XIRER2uPWcRgdFhbxKSs\nYWGEJiSYNpi//347TzwxAIAxY0b5b3PaqVM7Dh78PZCliSicRcScLrnkUl57bUKgyxApl8JZRILO\ngQP7ueWW6/jgg5n07Xsbqam3sX37Np566jF69bqesWOfJT19EykpvSpczuTJExk6dBBer7eWKhc5\nSuecRSQo5ebm4nDEMXv2AkaMGMzIkUOZNu19LBa49dYbaN26TYXvX7bsazZsWMfEiVOxWrUfI7VL\n4SwiQcnj8ZCU1A2ARo2aABATEwNAXFw8WVlZJ31vRsa/mTp1Em++OYnw8PCaL1bkT/R1UESCks1m\nIyysDgBWq5Xw8Lr+MavVSuvWl530vePGvYDFAlFRUTVep0h5FM4iIn8ycuRomjVrzttv64IxCQyF\ns4jInzRseD5PPDGYFSuWkp6+KdDliAkpnEVEyhEdHcNTTw3lhReeo7DwcKDLEZOx+Hw+X2XfVFJS\nwpAhQ9i/fz82m40XXniB888/v8w8LVu2pG3bP+7hO2PGDGwVPMg9MzO/smXUOKcz0pB11QSz9GqW\nPkG9BiOz9Anm6NXpjDzpWJWu1l60aBFRUVG8+uqrrF69mldffZXXX3+9zDwRERHMmjWrKosXEQkY\nr7cET0k+NnskVqs90OWISVUpnNetW0evXkd/vH/VVVcxbNiwai1KRKS2+XxecvZ9TVHuj3hKXNjs\n0YTHXExsg2uxWHQGUGpXlf6Ny8rKwvHfZ8JarVYsFgtut7vMPG63m0GDBpGamsr06dPPvFIRkRqU\ns+9rCjI34ilxAeApcVGQuZGcfV8HuDIxo1PuOaelpZGWllbmtS1btpSZLu+09eDBg+nZsycWi4W7\n7rqLdu3a0apVq5OuJza2LiEhJz8nHSgVnRMINmbp1Sx9gno9XV6Pm99+2FnumDt/J3GOW7DaQqu8\n/Oqkz9QcThnOycnJJCcnl3ltyJAhZGZm0rx5c0pKSvD5fISGlv0Xt2/fvv6/r7zySjIyMioM55yc\nwsrWXuPMcEHCMWbp1Sx9gnqtjJLibNxHcsodcx/J5bcDB7CHOaq8/OqizzS4VPTlo0qHtTt27MhX\nX30FwIoVK7jiiivKjO/evZtBgwbh8/koLS0lPT2dpk2bVmVVIiI1zmaPxGaPPslYNDa7effgJDCq\ndEHYDTfcwNq1a+nbty+hoaG8+OKLAEyZMoX27duTmJhI/fr16dOnD1arlaSkJFq3bl2thYuIVBer\n1U54zMUUZG48YSw8ppmu2pZaV6XfOdcEIx6+MMNhlWPM0qtZ+gT1Wll/XK2dcdzV2s0MdbW2PtPg\nUu2/cxYRCTYWixVHwx54z7tGv3OWgFM4i4gcx2q1YzXAxV9ibsY4ViMiIiJ+CmcRERGDUTiLiIgY\njMJZRETEYBTOIiIiBqNwFjkLpadvIiWlF5MmTeCTT+YB0KlTOw4e/D3AlYlIddBPqUTOYg89NCDQ\nJYhIDdCes8hZbMyYUcyY8c4Jr0+ePJH+/fvj9Xr56afdDBjwAKmpt/G3v6Xw739/H4BKRaQyFM4i\nQWbZsq/ZsGEdr7zyCgBDhz5Jjx43MGfOAp58cihDhgyitLQ0wFWKSEUUziJBJCPj30ydOomXXnqN\nunXr8ssvP5Obm82NN94CQOvWlxETE8v27VsDXKmIVETnnEWCyLhxL1C3bl2ioqIAKCjI58iRI9x5\nZx//PIcPH8blcgWqRBE5DQpnkSAycuRoPv10AW+/PYExY54lPt5JvXr1+PDD+YEuTUQqQYe1RYJI\nw4bn88QTg1mxYikbNmygfv1zcTrPYcWKpQDk5uYycuQwioqKAlypiFRE4SwSZKKjY3jqqaEMHTqU\noqJCnn12LPPnf8Qdd/RmwID7adfucsLDwwNdpohUwOLz+XyBLgIw5EO1zfCw72PM0qtZ+gT1GozM\n0ieYo1enM/KkY9pzFglyJSUeXDlFlJR4Al2KiJwmXRAmEqS8Hi+rl+7kp4wsCvKKiYgK46Jm8VyV\n1BirNXDfy91uN8uWfc31198UsBpEjE57ziJB6uvPvmfbpn0U5BUDUJBXzLZN+1i7/D8BrSsj40e+\n+uqLgNYgYnTacxYJQiUlHn7c/lu5Yz9nZHFFl0bY7bZqW9+XXy5i5sx3AWjZsiVPP/0Mixd/wZw5\n7+PxeIiLi+eZZ54jNDSU4cOf5PDhw/Tv/w/eeuvEW4+KiPacRYJSYYEbV275P5cqyC+msMBdbes6\ncGA/Eye+wYQJk5k9ez5FRUf46KMPGT/+ZcaPn8icOR/ToEFDZsx4B4cjjgcfHEDLlq0VzCIV0J6z\nSBCqGxFKdEw4rpwTAzoiMoy6EaHVtq6NG9fTqlVr4uOdwNEbodhsNlJS7sRutwPQpk0iixfrULbI\n6VI4iwQhu93GxZfWZ+O3P50w9pdm8dV6SNvlyiUi4o+fhISFheHxeJg69W3WrPkGj8dDYWEh559/\nQbWtUyTYKZxFgtS1N19CUZGbnzOyKMgvJiIyjL/892rt6hQdHVPmQRqHDxfwzTcrWbPmGyZMmEpM\nTAwLF37M119/Wa3rFQlmCmeRIGW1WenUrSlXdGlEYYGbuhGh1brHfEyHDh15++03OXBgP/Xrn8u4\ncS/QqFFj6tc/l5iYGFyuXJYvX+K/ZWhISAiFhQX4fD4sFku11yMSDHRBmEiQs9ttRMeG10gwAyQk\nnMPgwcN59NGH6dv3NiwWC9deez0ul4uUlF6MGjWc++/vz8GDv/Pmm+Np3foysrKy6NWrBx6Pbowi\nUh7dvrMCZrh93DFm6dUsfYJ6DUZm6RPM0atu3ykiAVdc4uFgTiHFuo2oyCnpnLOI1CiP18vc5bvY\nnJFJdl4xjqgwEps5SUlqgi2AtxEVMTKFs4jUqLnLd7F006/+6UN5xf7pO7o1C1RZIoamr60iUmOK\nSzxszsgsd2xzRpYOcYuchMJZRGqMq6CY7P8+eOPPcvKP4Coof0zE7BTOIlJjoiPCcESFlTsWG1mH\n6Ijyx0TMTuEsIjUmzG4jsZmz3LHEZvGE1dBvr0XOdrogTERqVEpSE+DoOeac/CPERtYhsVm8/3UR\nOZHCWURqlM1q5Y5uzejdpTGugmKiI8K0xyxyCgpnEakVYXYbCbF1A12GyFlB55xFREQMRuEsIiJi\nMApnERERg1E4i4iIGIzCWURExGAUziIiIgajcBYRETEYhbOIiIjBKJxFREQMRuEsIiJiMApnERER\ng6lyOG/cuJEOHTqwYsWKcscXLlxI7969SU5OJi0trcoFioiImE2VHnyxZ88epk+fTtu2bcsdLyws\nZOLEicybNw+73U6fPn3o3r07MTExZ1SsiIiIGVRpz9npdDJhwgQiIyPLHd+yZQutWrUiMjKSOnXq\n0LZtW9LT08+oUBEREbOo0p5zeHh4heNZWVk4HA7/tMPhIDMzsyqrEhERMZ1ThnNaWtoJ54wHDhxI\n586dT3slPp/vlPPExtYlJMR4D2B3Oss/OhCMzNKrWfoE9RqMzNInmKvXPztlOCcnJ5OcnFyphSYk\nJJCVleWfPnjwIJdddlmF78nJKazUOmqD0xlJZmZ+oMuoFWbp1Sx9gnoNRmbpE8zRa0VfPmrkp1Rt\n2rRh27Zt5OXlcfjwYdLT02nXrl1NrEpERCToVOmc88qVK5k2bRq7d+9mx44dzJo1i3fffZcpU6bQ\nvn17EhMTGTRoEP369cNisfDII4+c9OIxERERKcviO50TwrXAiIcvzHBY5Riz9GqWPkG9BiOz9Anm\n6LXWD2uLiIhI1SmcRUREDEbhLCIiYjAKZxEREYNROIuIiBiMwllERMRgFM4iIiIGo3AWERExGIWz\niIiIwSicRUREDEbhLCIiYjAKZxEREYNROIuIiBiMwllERMRgFM4iIiIGo3AWERExGIWziIiIwSic\nRUREDEbhLCIiYjAKZxEREYNROIuIiBiMwllERMRgFM4iIiIGo3AWERExGIWziIiIwSicRUREDEbh\nLCIiYjAKZxEREYNROIuIiBiMwllERMRgFM4iIiIGo3AWERExGIWziIiIwSicRUREDEbhLCIiZ2TL\nlu/o0+fmQJcRVBTOIiJnqeef/19Wr/6mWpb1xRef8dhj/atlWXLmQgJdgIiIVM0zzzwXsHXPmPEO\nCxd+THR0NJ06dQGguLiYf/3rVdLTN2G1Wrnyyo707/8oNpuNDRvW8dJLowkPD+f22+9g4sQ3mDlz\nNueee17AejAyhbOIiEF89tknzJnzPh6Ph7i4eJ555jnS0zexdu1q6tWrx44dWwELzz33Io0aNWbA\ngAe4+eZeXHfdDXTq1I7Bg4czb94cCgoKGD58FAsXfsz27Vv5y18a8dJLrxESEsLq1auYMuUtSkpK\nCA+vy9Chz9C06cWVqvOnn3Yzd+6HfPBBGtHRMYwY8TQAH300m4MHf2fWrI/weEoZMOABli5dTLdu\n1zFmzCiGDv1fOnToyMSJb3DkSFEN/BMMHjqsLSJiADk52Ywf/zLjx09kzpyPadCgITNmvAPA+vVr\nuPXWZBYvXkxiYjvS0maXuwyXK5f33ptLUlJ3hg8fTL9+DzJ79gJ2797Fd9+lU1payujRoxg8eASz\nZy+gc+cuTJjwRqVr3bIlncsua4vDEYfNZuO6664HYN261fTseSshISGEhdWhe/fr2bhxPXv37sHt\ndtOhQ0cAevdOwev1Vumfk1konEVEDCA21sHixatISDgHgDZtEtm/fx8Af/lLI5o3bwHAxRdfzO+/\n/1buMjp3vhqAxo2b0KBBQy644EJCQ0Np2PB8srIyCQkJYdGiJVx6aasT1lEZeXl5RERE+KcjI6MA\nyM3N8f999PVIcnJyyM/PK/N6fHx8pddpNjqsLSJiAB6Ph3femcSaNd/g8XgoLCzk/PMvAKBevT+C\n0Gq14fGUv9dZt27d/85jJTw8vMx7ju2ppqXN4auvFuF2u3G73VgslkrXGhkZRUFBgX86NzcHAIcj\nDpfL5X89L8+Fw+GgXr16FBUV+l/Pzj5U6XWajfacRUQMYNmyJaxZ8w0TJkxl9uwF9Ov3YLWvY9u2\nLXzwwUxefPE1Zs9ewJAhI6q0nEsvbcW2bd+Rk5ODx+Nh8eIvAbjqqk58/vmneDweioqKWLz4Czp0\n6ETDhhdQWlpKevomAD75ZH6VvhSYifacRUQMIDc3m/r1zyUmJgaXK5fly5dQVFS9F03l5OQQGxvL\nOefU58iRI3z55eccOVKEz+er1HKaNr2YW27pTb9+dxEVFU23bteye/cuevdOYf/+fdx99+1YLBa6\ndu1GUlI3LBYLTz45hLFjnyUiIpKUlDuwWq0K6AoonEVEDKBbt+tYsmQxKSm9OO+8Btx/f3+GDHmC\nCRNer/TV1CdzxRUd+PjjNFJSehEf7+SxxwaxY8c2RowYTMeOf63Usu6//2Huv/9h//Rdd90LwJNP\nDi13/m7drqNbt+sAKCoqYsyYUURERFatEROw+Cr7lamGZGbmB7qEEzidkYasqyaYpVez9AnqNRid\nzX3+4x9/o0/yHbS9/K+s/WYx89I+5P330046/9nc6+lyOk/+5UR7ziIiJuUtLqbU5SIkOhprWFiN\nrcfj9dKiQx9ee/MtSkpeJzQsnB69H8Dj9WKz6tKn8iicRURMxufxkJk2h4LN6ZRmZxPicBCR2BZn\ncioWm63a1zd3+S52ZEbQsNPj/te2HTj6+h3dmlX7+oKBvrKIiJhMZtoccpcuofTQIfD5KD10iNyl\nS8hMm1Pt6you8bA5I7Pcsc0ZWRSXeKp9ncFA4SwiYiLe4mIKNqeXO1aweTPe4uJqXZ+roJjsvPKX\nmZN/BFdB9a4vWFQ5nDdu3EiHDh1YsWJFueMtW7bk7rvv9v/P49G3IxGRQCt1uSjNzi5/LCeb0uNu\nIlIdoiPCcESVfz47NrIO0RE1d677bFalc8579uxh+vTptG3b9qTzREREMGvWrCoXJiIi1S8kOpoQ\nh+PoIe0/j8U6CImOrtb1hdltJDZzsnTTryeMJTaLJ8xe/ee4g0GV9pydTicTJkwgMlK/URMROZtY\nw8KISCx/xyoiMbFGrtpOSWpCt3YNiYuqg9UCcVF16NauISlJTap9XcGiSnvOx9+z9WTcbjeDBg1i\n3759XHfdddx3331VWZWIiFSuL74qAAAU4UlEQVQzZ3IqcPQcc2lONiGxDiISE/2vVzeb1cod3ZrR\nu0tjXAXFREeEaY/5FE4ZzmlpaaSllf2h+MCBA+ncuXOF7xs8eDA9e/bEYrFw11130a5dO1q1anXS\n+WNj6xISYrwPq6IfiQcbs/Rqlj5BvQaj6uozYeBDeIqLcWfnEOqIxVaDv3M+XsNKzGuWz7Q8pwzn\n5ORkkpOTK73gvn37+v++8sorycjIqDCcc3IKTzoWKGa4Q80xZunVLH2Ceg1GNdJnSD3IcwPu6l3u\nGTLDZ1rRl48a+SnV7t27GTRoED6f779PIkmnadOmNbEqERGRoFOlc84rV65k2rRp7N69mx07djBr\n1izeffddpkyZQvv27UlMTKR+/fr06dMHq9VKUlISrVu3ru7aRUREgpIefFEBMxxWOcYsvZqlT1Cv\nwcgsfYI5eq31w9oiIiJSdQpnERERg1E4i4iIGIzCWURExGAUziIiIgajcBYRETEYhbOIiIjBKJxF\nREQMRuEsIiJiMApnERERg1E4i4iIGIzCWURExGAUziIiIgajcBYRETEYhbOIiIjBKJxFRCRopKdv\nIiWlV42v5/vvt/PEEwNqbPkKZxERkUq65JJLee21CTW2/JAaW7KIiMhp+uyzT5gz5308Hg9xcfGM\nH/8qs2bN5rffDuBy5fKf/+wiISGBsWNfITbWQZ8+N3PrrX1YtmwJBw/+xi239Ob++x8us0y3281b\nb73B+vXrKC0toWfPW/nb3/5eLfWmp2/ipZdGM3fuJ9WyvD/TnrOIiARUTk4248e/zPjxE5kz52Ma\nNGjIW2+9BcCqVSt4/PGnmD9/Eeed14BZs6b737d9+zamTp3JrFkfsWBBGjt3ZpRZ7ocfvsdPP/3E\ne+/NYdasj1i5chlr1nxbq71VlcJZREQCKjbWweLFq0hIOAeANm0S2bt3LwBt2/4P553XAIC//jWJ\nbdu2+t/Xo8eN2Gw2YmMdtGlzGdu3by2z3DVrvuG22/oQGhpKeHg4PXrcyKpVy2upqzOjw9oiIhJQ\nHo+Hd96ZxJo13+DxeCgsLKRx40YAREVF++eLjIwkPz/fPx0VFXXcWBT5+XlllpufX8C//vUakydP\nBKCkpIQWLVrWZCvVRuEsIiIBtWzZEtas+YYJE6YSExPDwoUfs3LlEgByc3P98+Xn55UJZJfrj7G8\nPFeZMYD4+Hj69r2bjh0713AH1U+HtUVEJKByc7OpX/9cYmJicLlyWb58CYcPHwZg69bv+P333wBY\nsWIZbdok+t+3bNkSvF4v2dmH2Lp1C61bJ5ZZbufOXVi06BM8Hg8+n48ZM95h/fq1tdfYGdCes4iI\nBFS3btexZMliUlJ6cd55Dbj//v4MGzaIwsJC2re/gtdee4mdOzM455z6PP74U/73XXRRI+6//x5+\n+20/ffqk0KhRY9LTN/nHb7vtdg4cOMDdd9+Oz+ejefNLuP32OwLRYqVZfD6fL9BFAGRm5p96plrm\ndEYasq6aYJZezdInqNdgZJY+4WivL774CpmZBxky5JkTxvv0uZlnnnmeNm0uC0B11cPpjDzpmA5r\ni4hIUCop8eDKKaKkxHPGy3J73GQWHsLtcVdDZaemw9oiIhJUvF4va5f/h58ysijIKyYiKoyLmsVz\nVVJjrNbK7ZN6vB4W7PqcrZk7yCnOJTYshtbOltzW5EZsVlsNdaBwFhERg+rX78GTjs2b99lJx9Yu\n/w/bNu3zTxfkFfunO3VrWqkaFuz6nJW/rvZPZxfn+KeTm/Ws1LIqQ4e1RUQkaJSUePgpI6vcsZ8z\nsip1iNvtcbM1c0e5Y9uydtToIW6Fs4iIBI3CAjcFecXljhXkF1NYcPqB6irOJ6c4t9yx7CO5uIpr\n7uI8hbOIiASNuhGhRESFlTsWERlG3YjQ015WdFgksWEx5Y456sQQHXbyq63PlMJZRESCht1u46Jm\n8eWO/aVZPHb76V/EFWoLpbWz/Nt9topvSajt9IO+snRBmIiI1Jpvv13J1KlvU1R0hIYNG/K3v/2d\nYcOeYuHCxdjtdgBGjBhMhw5X0L37zSd95OP8+XNZsCANn89HvXr1GDp0JI0aNQbgqqSj//9zRhYF\n+cVERIbxl/9erV1ZtzW5ETh6jjn7SC6OOjG0im/pf72mKJxFRKRW7Nv3K88/P5JJk6bRqFETZs2a\nzvz5c0lISGDDhrV06tSF4uJiNm7cwHPPjeK99/545KPH4+GRR/5B48ZNSUxsy9Spk1iwYBF169Zj\n+fKlrFu32h/OVquVTt2ackWXRhQWuKkbEVqpPebj2aw2kpv15JbGPXAV5xMdFlmje8zH6LC2iIjU\nig0b1pGY2JZGjZoAcMstvVm9+huSkrqzZMliAP7f/1tPs2YXc84555z0kY+hoWFYLBYWLfqU7OxD\nJCV148477zlhfXa7jejY8CoH8/FCbaE468bVSjCD9pxFRKSWFBTks2XLZu64o7f/tYiICNq1u4L3\n359BUVER33yzkqSk7sDJH/kYEhLCG2+8xXvvTWfatMk0btyUQYOG0Lhxk4D0VRMUziIiUivi4520\na3c5o0e/fMJYo0ZN+Pbblaxdu5qHHx743/lP/sjHZs2aM3r0S5SUlPDBBzN55ZWxvP32uzXeQ23R\nYW0REakVl1/egS1bvmPfvl8B+P777bz++isAdO9+HVOmvEXjxk2JjXUAJ3/k43/+s4sRI56mpKQE\nu91O8+aXAJZAtVUjtOcsIiK1Ij4+nqefHs6wYU9RWlpC3bp1efTRQQAkJXXnzTfH+6/GhpM/8jE8\nPJzzzjuPu+++nZAQO3Xr1uWJJ54OVFs1Qo+MrIDZHs9mhl7N0ieo12AUzH263W6Sk29m1qyPiIqK\nDupej9EjI0VExNDmzv2QDh06ERUVfVrzuz1eDh1x4/Z4a7iywNBhbRERCag77uhNbKyDMWPGnXJe\nj8/Hl3uy+CG3gFx3KTGhIbSIieD6C+KxWYLnvLPCWUREAurDD+ef9rxf7sli7cE/HkaR4y71T990\nobPaawsUHdYWEZGzgtvj5YfcgnLHfsgtCKpD3ApnERE5K+SXlJLrLi13LNddSn5J+WNnI4WziIic\nFSLtIcSEln82NiY0hEh78JypVTiLiMhZIdRmpUVMRLljLWIiCLUFT6QFz9cMEREJetdfcPRZzeVd\nrR1MFM4iInLWsFks3HShk2sbxpFfUkqkPSSo9piPqVI4l5aWMnz4cPbs2YPH42Hw4MG0a9euzDwL\nFy5k5syZWK1Wbr/9dpKTk6ulYBERkVCblbhaenxjIFQpnD/99FPCw8OZPXs2O3fuZOjQocybN88/\nXlhYyMSJE5k3bx52u50+ffrQvXt3YmJiqq1wERGRYFWlYwE9e/Zk6NChADgcDnJzc8uMb9myhVat\nWhEZGUmdOnVo27Yt6enpZ16tiIiICVRpz9lut/v/njlzJjfddFOZ8aysLBwOh3/a4XCQmZlZxRJF\nRETM5ZThnJaWRlpaWpnXBg4cSOfOnfnggw/YsWMHkyZNqnAZp/Pgq9jYuoSE2E45X22r6KkhwcYs\nvZqlT1CvwcgsfYK5ev2zU4ZzcnJyuRdzpaWlsXz5ct56660ye9IACQkJZGVl+acPHjzIZZddVuF6\ncnIKT7fmWmOGR5YdY5ZezdInqNdgZJY+wRy9VvsjI/fu3cucOXOYMGECYWFhJ4y3adOGbdu2kZeX\nx+HDh0lPTz/ham4REREpX5XOOaelpZGbm8sDDzzgf23atGnMmDGD9u3bk5iYyKBBg+jXrx8Wi4VH\nHnmEyEjzHp4QERGpDIvvdE4I1wIjHr4ww2GVY8zSq1n6BPUajMzSJ5ij12o/rC0iIiI1R+EsIiJi\nMApnERERg1E4i4iIGIzCWURExGAUziIiIgajcBYRETEYhbOIiIjBKJxFREQMRuEsIiJiMApnERER\ng1E4i4hIjTpwYD9dulxRqffs3r2b775Lr6GKjE/hLCIihrN06VJTh3OVHhkpIiJSWYsWfUpa2mzy\n8/N5+OGBXHPNtYwfP45NmzZQWlpK69ZtGDp0JOvXr2Xy5MmEhISQn5/PwIH/5NNPFzB37ge43W5a\ntmzFsGH/S1hYnUC3VGMUziIiUuO8Xi+lpSXMnDmHFSuW8vbbb2K329m6dTOzZn2Ex+OhX7+7WLbs\na6677ga6d+9OXNw53HvvP9iyZTPvvDOJ6dM/ID7eybhxY5k6dRIDBjwe6LZqjA5ri4hIjfP5fPTo\ncRMAzZo1JzPzIFdffQ3vvDOLkJAQwsLCaN78Evbv33fCe9es+YZrrulOfLwTgF69erNq1Yparb+2\nac9ZRERqnM1mo06do4ehrVYrXq+XnJwcXn/9ZX788UesVgvZ2YdITu57wnvz8wv49tsVbNy4HgCv\n10dpaUmt1l/bFM4iIhIQU6a8RUhICO+9N4fQ0FCefXZEufPFx8fTo8dNQX0Y+890WFtERAIiNzeb\nRo2aEBoays6dGWzbtoWioiIAQkJCKCgoAKBTpy6sWrWCnJwcAL79diXvvz8jUGXXCu05i4hIQKSm\n3sXo0aP44ovPaN06kQEDHufFF5/nkksupWvXrgwaNIjfftvP6NEv87e/3cfAgQ/i83mJjXXw1FPD\nAl1+jbL4fD5foIsAyMzMD3QJJ3A6Iw1ZV00wS69m6RPUazAyS59gjl6dzsiTjumwtoiIGJ7b4+XQ\nETdujzfQpdQKHdYWERHD8vh8fLknix9yC8h1lxITGkKLmAiuvyAem8US6PJqjMJZREQM68s9Waw9\nmOufznGX+qdvutAZqLJqnA5ri4iIIbk9Xn7ILSh37IfcgqA+xK1wFhERQ8ovKSXXXVruWK67lPyS\n8seCgcJZREQMKdIeQkxo+WdfY0JDiLQH75lZhbOIiBhSqM1Ki5iIcsdaxEQQagveCAverx0iInLW\nu/6CeIByr9YOZgpnERExLJvFwk0XOrm2YRz5JaVE2kOCeo/5GIWziIgYXqjNSpwtNNBl1Jrg//oh\nIiJyllE4i4iIGIzCWURExGAUziIiIgajcBYRETEYhbOIiIjBKJxFREQMRuEsIiJiMApnERERg1E4\ni4iIGIzF5/P5Al2EiIiI/EF7ziIiIgajcBYRETEYhbOIiIjBKJxFREQMRuEsIiJiMApnERERgwkJ\ndAGBVlpayvDhw9mzZw8ej4fBgwfTrl27MvN88cUXvPvuu1itVjp06MA///lPFixYwBtvvMEFF1wA\nwFVXXcXDDz8ciBZOy+n0uXDhQmbOnInVauX2228nOTmZkpIShgwZwv79+7HZbLzwwgucf/75Aeri\n9G3cuJHHHnuMsWPH0rVr1zJj27dv56WXXvJP79q1i4kTJ7JmzRo+++wzzjnnHAB69uxJcnJyrdZd\nFRX1CtCyZUvatm3rn54xYwZer/es+1xP1WcwbKfHnKrXYNlWT1VzsG2rleIzuXnz5vlGjhzp8/l8\nvoyMDF/v3r3LjBcWFvq6du3qy8/P93m9Xl+fPn18O3fu9M2fP9/34osvBqDiqjlVn4cPH/Zde+21\nvry8PF9RUZHvxhtv9OXk5PgWLFjgGzVqlM/n8/m+/fZb32OPPVbbpVfaL7/84nvooYd8/fv39y1f\nvrzCeV0ul+/OO+/0eTwe37/+9S/frFmzaqnK6nE6vV5++eUnvHa2fa6n6jNYtlOf79S9BtO2Wpma\nz/ZttbJMf1i7Z8+eDB06FACHw0Fubm6Z8fDwcBYuXEhERAQWi4WYmJgT5jkbnKrPLVu20KpVKyIj\nI6lTpw5t27YlPT2ddevW0b17d+DoXkd6enqt115ZTqeTCRMmEBkZecp5p02bxj333IPVenZuCpXp\n9Xhn2+d6qj6DZTuFU/caTNtqZWo+27fVyjJHlxWw2+2EhYUBMHPmTG666aYT5omIiADgxx9/ZN++\nfbRp0wY4euipX79+3HPPPXz//fe1V3QVnKrPrKwsHA6Hf9rhcJCZmVnmdavVisViwe12117hVRAe\nHo7NZjvlfEeOHGH16tVcc801/te++uor7rvvPh588EH27t1bk2VWi9Pp1e12M2jQIFJTU5k+fTrA\nWfe5nk6fwbCdwql7DaZt9XRrDoZttbJMdc45LS2NtLS0Mq8NHDiQzp0788EHH7Bjxw4mTZpU7nt/\n/vlnnnzySV599VXsdjtt2rTB4XBw9dVXs3nzZp5++mk+++yz2mjjlM6kz2N8J7mr68leD5SKej2V\npUuXcvXVV/u/iXfp0oUrr7yS9u3b8/nnnzN69GgmT55cI3VXRVV7HTx4MD179sRisXDXXXedcK0B\nGOtzPZPP9GzaTuHMej3mbN5Wt2zZUmb6ZDWfbdtqdTBVOCcnJ5d70UBaWhrLly/nrbfewm63nzD+\n22+/8cgjj/Dyyy/TokULABo3bkzjxo0BSExMJDs7G4/Hc1p7bDWtKn0mJCSQlZXlnz548CCXXXYZ\nCQkJZGZm0rx5c0pKSvD5fISGhtZ4D6frZL2ejhUrVtC3b1//dOvWrf1/JyUl8corr5xxfdWpqr0e\n3+OVV15JRkaGoT/XqvZ5tm2nULVeg2lbHTJkyGnVfLZtq9XB9Ie19+7dy5w5c5gwYYL/sO+fDR8+\nnFGjRtGyZUv/a1OnTmXRokUAZGRk4HA4DLPBl+dUfbZp04Zt27aRl5fH4cOHSU9Pp127dnTs2JGv\nvvoKOLqBXHHFFbVdeo3Zvn07zZs390+PHj2aTZs2AUcPhTZt2jRQpVWb3bt3M2jQIHw+H6WlpaSn\np9O0adOg/FyDYTs9HcG0rZ5uzWbYVv/M9E+leu211/j8888577zz/K9NmzaNGTNm0L59e2JiYujV\nq1eZb2r33nsvLVu25KmnnvL/R2/YsGFl5jGaU/WZmJjIV199xbRp0/yHP3v27InH42HEiBH8/PPP\nhIaG8uKLL3LuuecGsJNTW7lyJdOmTWP37t04HA6cTifvvvsuU6ZM8fcK0KFDB9atW+d/348//sjI\nkSMJCQnBYrEwevRoLrzwwkC1cVpOp9dx48axfv16rFYrSUlJPPzww2fd53qqPoNlO4XT+0yDZVs9\nWc3BuK1WlunDWURExGhMf1hbRETEaBTOIiIiBqNwFhERMRiFs4iIiMEonEVERAxG4SwiImIwCmcR\nERGDUTiLiIgYzP8HGchs4S7MCOEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x396 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "'''\n",
    "  code by Tae Hwan Jung(Jeff Jung) @graykode\n",
    "  reference : https://github.com/golbin/TensorFlow-Tutorials/blob/master/04%20-%20Neural%20Network%20Basic/03%20-%20Word2Vec.py\n",
    "'''\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "# 3 Words Sentence\n",
    "sentences = [ \"i like dog\", \"i like cat\", \"i like animal\",\n",
    "              \"dog cat animal\", \"apple cat dog like\", \"dog fish milk like\",\n",
    "              \"dog cat eyes like\", \"i like apple\", \"apple i hate\",\n",
    "              \"apple i movie book music like\", \"cat dog hate\", \"cat dog like\"]\n",
    "\n",
    "word_sequence = \" \".join(sentences).split()\n",
    "word_list = \" \".join(sentences).split()\n",
    "word_list = list(set(word_list))\n",
    "word_dict = {w: i for i, w in enumerate(word_list)}\n",
    "\n",
    "# Word2Vec Parameter\n",
    "batch_size = 20\n",
    "embedding_size = 2 # To show 2 dim embedding graph\n",
    "num_sampled = 10 # for negative sampling, less than batch_size\n",
    "voc_size = len(word_list)\n",
    "\n",
    "def random_batch(data, size):\n",
    "    random_inputs = []\n",
    "    random_labels = []\n",
    "    random_index = np.random.choice(range(len(data)), size, replace=False)\n",
    "\n",
    "    for i in random_index:\n",
    "        random_inputs.append(data[i][0])  # target\n",
    "        random_labels.append([data[i][1]])  # context word\n",
    "\n",
    "    return random_inputs, random_labels\n",
    "\n",
    "# Make skip gram of one size window\n",
    "skip_grams = []\n",
    "for i in range(1, len(word_sequence) - 1):\n",
    "    target = word_dict[word_sequence[i]]\n",
    "    context = [word_dict[word_sequence[i - 1]], word_dict[word_sequence[i + 1]]]\n",
    "\n",
    "    for w in context:\n",
    "        skip_grams.append([target, w])\n",
    "\n",
    "# Model\n",
    "inputs = tf.placeholder(tf.int32, shape=[batch_size])\n",
    "labels = tf.placeholder(tf.int32, shape=[batch_size, 1]) # To use tf.nn.nce_loss, [batch_size, 1]\n",
    "\n",
    "embeddings = tf.Variable(tf.random_uniform([voc_size, embedding_size], -1.0, 1.0))\n",
    "selected_embed = tf.nn.embedding_lookup(embeddings, inputs)\n",
    "\n",
    "nce_weights = tf.Variable(tf.random_uniform([voc_size, embedding_size], -1.0, 1.0))\n",
    "nce_biases = tf.Variable(tf.zeros([voc_size]))\n",
    "\n",
    "# Loss and optimizer\n",
    "cost = tf.reduce_mean(tf.nn.nce_loss(nce_weights, nce_biases, labels, selected_embed, num_sampled, voc_size))\n",
    "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n",
    "\n",
    "# Training\n",
    "with tf.Session() as sess:\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(5000):\n",
    "        batch_inputs, batch_labels = random_batch(skip_grams, batch_size)\n",
    "        _, loss = sess.run([optimizer, cost], feed_dict={inputs: batch_inputs, labels: batch_labels})\n",
    "\n",
    "        if (epoch + 1) % 1000 == 0:\n",
    "            print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n",
    "\n",
    "    trained_embeddings = embeddings.eval()\n",
    "\n",
    "for i, label in enumerate(word_list):\n",
    "    x, y = trained_embeddings[i]\n",
    "    plt.scatter(x, y)\n",
    "    plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "Word2Vec-Tensor(NCE_loss).ipynb",
   "version": "0.3.2",
   "provenance": []
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "accelerator": "GPU"
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
